{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[0.5, 0.5, 0. , 0. , 0. ],\n",
       "        [0.5, 0. , 0.5, 0. , 0. ],\n",
       "        [0. , 0. , 0. , 0.5, 0.5],\n",
       "        [0. , 0.1, 0.2, 0.2, 0.5],\n",
       "        [0. , 0. , 0. , 0. , 0. ]]),\n",
       " array([[  -1.,    0., -100., -100., -100.],\n",
       "        [  -1., -100.,   -2., -100., -100.],\n",
       "        [-100., -100., -100.,   -2.,    0.],\n",
       "        [-100.,    1.,    1.,    1.,   10.],\n",
       "        [-100., -100., -100., -100., -100.]]))"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "#状态转移概率矩阵\n",
    "#很显然,状态4(第5行)就是重点了,要进入状态4,只能从状态2,3进入\n",
    "#[5, 5]\n",
    "P = np.array([\n",
    "    [0.5, 0.5, 0.0, 0.0, 0.0],\n",
    "    [0.5, 0.0, 0.5, 0.0, 0.0],\n",
    "    [0.0, 0.0, 0.0, 0.5, 0.5],\n",
    "    [0.0, 0.1, 0.2, 0.2, 0.5],\n",
    "    [0.0, 0.0, 0.0, 0.0, 0.0],\n",
    "])\n",
    "\n",
    "#反馈矩阵，-100的位置是不可能走到的\n",
    "#[5, 5]\n",
    "R = np.array([\n",
    "    [-1.0, 0.0, -100.0, -100.0, -100.0],\n",
    "    [-1.0, -100.0, -2.0, -100.0, -100.0],\n",
    "    [-100.0, -100.0, -100.0, -2.0, 0.0],\n",
    "    [-100.0, 1.0, 1.0, 1.0, 10.0],\n",
    "    [-100.0, -100.0, -100.0, -100.0, -100.0],\n",
    "])\n",
    "\n",
    "P, R"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([2, 3, 4], [-2.0, 10.0])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "\n",
    "\n",
    "#生成一个chain\n",
    "def get_chain(max_lens):\n",
    "    #采样结果\n",
    "    ss = []\n",
    "    rs = []\n",
    "\n",
    "    #随机选择一个除4以外的状态作为起点\n",
    "    s = random.choice(range(4))\n",
    "    ss.append(s)\n",
    "\n",
    "    for _ in range(max_lens):\n",
    "        #按照P的概率，找到下一个状态\n",
    "        s_next = np.random.choice(np.arange(5), p=P[s])\n",
    "\n",
    "        #取到r\n",
    "        r = R[s, s_next]\n",
    "\n",
    "        #s_next变成当前状态,开始接下来的循环\n",
    "        s = s_next\n",
    "\n",
    "        ss.append(s)\n",
    "        rs.append(r)\n",
    "\n",
    "        #如果状态到了4则结束\n",
    "        if s == 4:\n",
    "            break\n",
    "\n",
    "    return ss, rs\n",
    "\n",
    "\n",
    "get_chain(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([[2, 4],\n",
       "  [0, 0, 0, 1, 0, 1, 2, 3, 4],\n",
       "  [0, 1, 0, 1, 0, 0, 1, 2, 3, 4],\n",
       "  [1, 2, 4],\n",
       "  [3, 4],\n",
       "  [1, 2, 4],\n",
       "  [1, 0, 1, 0, 0, 1, 2, 3, 1, 0, 0, 0, 1, 0, 0, 0, 1, 2, 4],\n",
       "  [2, 4],\n",
       "  [2, 3, 4],\n",
       "  [2, 4],\n",
       "  [0, 1, 0, 0, 0, 1, 2, 4],\n",
       "  [3, 4],\n",
       "  [1, 2, 3, 1, 0, 1, 0, 1, 2, 4],\n",
       "  [2, 3, 3, 3, 2, 4],\n",
       "  [1, 2, 4],\n",
       "  [0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 2, 4],\n",
       "  [0, 1, 2, 4],\n",
       "  [2, 4],\n",
       "  [2, 4],\n",
       "  [2, 4],\n",
       "  [2, 4],\n",
       "  [1, 2, 4],\n",
       "  [2, 4],\n",
       "  [3, 2, 3, 1, 2, 3, 4],\n",
       "  [0, 0, 1, 2, 3, 3, 3, 4],\n",
       "  [2, 3, 2, 4],\n",
       "  [2, 4],\n",
       "  [2, 4],\n",
       "  [1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0],\n",
       "  [1, 2, 4],\n",
       "  [2, 3, 3, 4],\n",
       "  [0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 2, 3, 4],\n",
       "  [1, 2, 3, 4],\n",
       "  [0, 1, 2, 3, 4],\n",
       "  [1, 0, 0, 0, 0, 1, 0, 1, 2, 3, 1, 2, 4],\n",
       "  [2, 3, 4],\n",
       "  [2, 4],\n",
       "  [2, 4],\n",
       "  [3, 3, 1, 2, 3, 2, 3, 4],\n",
       "  [3, 4],\n",
       "  [0, 1, 2, 3, 3, 4],\n",
       "  [3, 4],\n",
       "  [0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 2, 3, 4],\n",
       "  [3, 3, 4],\n",
       "  [2, 3, 4],\n",
       "  [1, 0, 1, 0, 1, 2, 3, 4],\n",
       "  [0, 1, 2, 3, 3, 3, 4],\n",
       "  [1, 0, 0, 0, 1, 0, 0, 0, 1, 2, 3, 4],\n",
       "  [3, 4],\n",
       "  [2, 3, 3, 4],\n",
       "  [2, 4],\n",
       "  [1, 2, 3, 3, 3, 2, 3, 3, 4],\n",
       "  [0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 2, 4],\n",
       "  [2, 4],\n",
       "  [1, 2, 3, 4],\n",
       "  [2, 4],\n",
       "  [3, 2, 4],\n",
       "  [1, 0, 1, 2, 4],\n",
       "  [3, 3, 3, 3, 4],\n",
       "  [1, 0, 1, 2, 4],\n",
       "  [0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 2, 3, 4],\n",
       "  [3, 4],\n",
       "  [0, 0, 0, 1, 2, 4],\n",
       "  [0, 1, 0, 0, 0, 1, 2, 4],\n",
       "  [2, 4],\n",
       "  [0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 2, 4],\n",
       "  [2, 4],\n",
       "  [3, 3, 3, 4],\n",
       "  [0, 0, 0, 1, 2, 4],\n",
       "  [2, 3, 2, 4],\n",
       "  [3, 4],\n",
       "  [2, 3, 4],\n",
       "  [0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 2, 4],\n",
       "  [2, 4],\n",
       "  [1, 0, 1, 2, 4],\n",
       "  [3, 4],\n",
       "  [1, 0, 1, 0, 0, 0, 0, 1, 2, 3, 3, 3, 2, 3, 3, 1, 2, 4],\n",
       "  [2, 3, 4],\n",
       "  [0, 1, 0, 0, 1, 2, 3, 4],\n",
       "  [0, 0, 1, 2, 4],\n",
       "  [3, 2, 3, 4],\n",
       "  [0, 0, 1, 2, 3, 3, 4],\n",
       "  [0, 1, 2, 4],\n",
       "  [0, 0, 1, 2, 4],\n",
       "  [0, 0, 1, 0, 0, 0, 1, 0, 1, 2, 4],\n",
       "  [3, 4],\n",
       "  [3, 1, 2, 4],\n",
       "  [2, 3, 2, 4],\n",
       "  [1, 2, 4],\n",
       "  [0, 1, 2, 4],\n",
       "  [1, 0, 0, 0, 0, 1, 2, 3, 3, 3, 3, 4],\n",
       "  [3, 3, 4],\n",
       "  [3, 3, 4],\n",
       "  [3, 3, 2, 3, 4],\n",
       "  [0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 4],\n",
       "  [3, 2, 3, 2, 3, 2, 3, 1, 0, 0, 1, 2, 4],\n",
       "  [2, 3, 1, 2, 4],\n",
       "  [1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 2, 4],\n",
       "  [3, 3, 4],\n",
       "  [3, 4]],\n",
       " [[0.0],\n",
       "  [-1.0, -1.0, 0.0, -1.0, 0.0, -2.0, -2.0, 10.0],\n",
       "  [0.0, -1.0, 0.0, -1.0, -1.0, 0.0, -2.0, -2.0, 10.0],\n",
       "  [-2.0, 0.0],\n",
       "  [10.0],\n",
       "  [-2.0, 0.0],\n",
       "  [-1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -2.0,\n",
       "   -2.0,\n",
       "   1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -2.0,\n",
       "   0.0],\n",
       "  [0.0],\n",
       "  [-2.0, 10.0],\n",
       "  [0.0],\n",
       "  [0.0, -1.0, -1.0, -1.0, 0.0, -2.0, 0.0],\n",
       "  [10.0],\n",
       "  [-2.0, -2.0, 1.0, -1.0, 0.0, -1.0, 0.0, -2.0, 0.0],\n",
       "  [-2.0, 1.0, 1.0, 1.0, 0.0],\n",
       "  [-2.0, 0.0],\n",
       "  [-1.0, -1.0, 0.0, -1.0, -1.0, -1.0, 0.0, -1.0, 0.0, -2.0, 0.0],\n",
       "  [0.0, -2.0, 0.0],\n",
       "  [0.0],\n",
       "  [0.0],\n",
       "  [0.0],\n",
       "  [0.0],\n",
       "  [-2.0, 0.0],\n",
       "  [0.0],\n",
       "  [1.0, -2.0, 1.0, -2.0, -2.0, 10.0],\n",
       "  [-1.0, 0.0, -2.0, -2.0, 1.0, 1.0, 10.0],\n",
       "  [-2.0, 1.0, 0.0],\n",
       "  [0.0],\n",
       "  [0.0],\n",
       "  [-1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -1.0],\n",
       "  [-2.0, 0.0],\n",
       "  [-2.0, 1.0, 10.0],\n",
       "  [-1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -2.0,\n",
       "   -2.0,\n",
       "   10.0],\n",
       "  [-2.0, -2.0, 10.0],\n",
       "  [0.0, -2.0, -2.0, 10.0],\n",
       "  [-1.0, -1.0, -1.0, -1.0, 0.0, -1.0, 0.0, -2.0, -2.0, 1.0, -2.0, 0.0],\n",
       "  [-2.0, 10.0],\n",
       "  [0.0],\n",
       "  [0.0],\n",
       "  [1.0, 1.0, -2.0, -2.0, 1.0, -2.0, 10.0],\n",
       "  [10.0],\n",
       "  [0.0, -2.0, -2.0, 1.0, 10.0],\n",
       "  [10.0],\n",
       "  [-1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -2.0,\n",
       "   -2.0,\n",
       "   10.0],\n",
       "  [1.0, 10.0],\n",
       "  [-2.0, 10.0],\n",
       "  [-1.0, 0.0, -1.0, 0.0, -2.0, -2.0, 10.0],\n",
       "  [0.0, -2.0, -2.0, 1.0, 1.0, 10.0],\n",
       "  [-1.0, -1.0, -1.0, 0.0, -1.0, -1.0, -1.0, 0.0, -2.0, -2.0, 10.0],\n",
       "  [10.0],\n",
       "  [-2.0, 1.0, 10.0],\n",
       "  [0.0],\n",
       "  [-2.0, -2.0, 1.0, 1.0, 1.0, -2.0, 1.0, 10.0],\n",
       "  [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 0.0, -1.0, 0.0, -1.0, 0.0, -2.0, 0.0],\n",
       "  [0.0],\n",
       "  [-2.0, -2.0, 10.0],\n",
       "  [0.0],\n",
       "  [1.0, 0.0],\n",
       "  [-1.0, 0.0, -2.0, 0.0],\n",
       "  [1.0, 1.0, 1.0, 10.0],\n",
       "  [-1.0, 0.0, -2.0, 0.0],\n",
       "  [-1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -2.0,\n",
       "   -2.0,\n",
       "   10.0],\n",
       "  [10.0],\n",
       "  [-1.0, -1.0, 0.0, -2.0, 0.0],\n",
       "  [0.0, -1.0, -1.0, -1.0, 0.0, -2.0, 0.0],\n",
       "  [0.0],\n",
       "  [-1.0, -1.0, -1.0, 0.0, -1.0, 0.0, -1.0, 0.0, -1.0, 0.0, -2.0, 0.0],\n",
       "  [0.0],\n",
       "  [1.0, 1.0, 10.0],\n",
       "  [-1.0, -1.0, 0.0, -2.0, 0.0],\n",
       "  [-2.0, 1.0, 0.0],\n",
       "  [10.0],\n",
       "  [-2.0, 10.0],\n",
       "  [-1.0, -1.0, 0.0, -1.0, 0.0, -1.0, -1.0, 0.0, -1.0, -1.0, 0.0, -2.0, 0.0],\n",
       "  [0.0],\n",
       "  [-1.0, 0.0, -2.0, 0.0],\n",
       "  [10.0],\n",
       "  [-1.0,\n",
       "   0.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   -1.0,\n",
       "   0.0,\n",
       "   -2.0,\n",
       "   -2.0,\n",
       "   1.0,\n",
       "   1.0,\n",
       "   1.0,\n",
       "   -2.0,\n",
       "   1.0,\n",
       "   1.0,\n",
       "   -2.0,\n",
       "   0.0],\n",
       "  [-2.0, 10.0],\n",
       "  [0.0, -1.0, -1.0, 0.0, -2.0, -2.0, 10.0],\n",
       "  [-1.0, 0.0, -2.0, 0.0],\n",
       "  [1.0, -2.0, 10.0],\n",
       "  [-1.0, 0.0, -2.0, -2.0, 1.0, 10.0],\n",
       "  [0.0, -2.0, 0.0],\n",
       "  [-1.0, 0.0, -2.0, 0.0],\n",
       "  [-1.0, 0.0, -1.0, -1.0, -1.0, 0.0, -1.0, 0.0, -2.0, 0.0],\n",
       "  [10.0],\n",
       "  [1.0, -2.0, 0.0],\n",
       "  [-2.0, 1.0, 0.0],\n",
       "  [-2.0, 0.0],\n",
       "  [0.0, -2.0, 0.0],\n",
       "  [-1.0, -1.0, -1.0, -1.0, 0.0, -2.0, -2.0, 1.0, 1.0, 1.0, 10.0],\n",
       "  [1.0, 10.0],\n",
       "  [1.0, 10.0],\n",
       "  [1.0, 1.0, -2.0, 10.0],\n",
       "  [-1.0, -1.0, 0.0, -1.0, -1.0, -1.0, -1.0, 0.0, -2.0, 0.0],\n",
       "  [1.0, -2.0, 1.0, -2.0, 1.0, -2.0, 1.0, -1.0, -1.0, 0.0, -2.0, 0.0],\n",
       "  [-2.0, 1.0, -2.0, 0.0],\n",
       "  [-1.0, -1.0, -1.0, -1.0, -1.0, 0.0, -1.0, -1.0, 0.0, -2.0, 0.0],\n",
       "  [1.0, 10.0],\n",
       "  [10.0]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#生成N个chain\n",
    "def get_chains(N, max_lens):\n",
    "    ss = []\n",
    "    rs = []\n",
    "    for _ in range(N):\n",
    "        s, r = get_chain(max_lens)\n",
    "        ss.append(s)\n",
    "        rs.append(r)\n",
    "\n",
    "    return ss, rs\n",
    "\n",
    "\n",
    "ss, rs = get_chains(100, 20)\n",
    "\n",
    "ss, rs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#给定一条链,计算回报\n",
    "def get_value(rs):\n",
    "    sum = 0\n",
    "    for i, r in enumerate(rs):\n",
    "        #给每一步的反馈做一个系数,随着步数往后衰减,也就是说,越早的动作影响越大\n",
    "        sum += 0.5**i * r\n",
    "\n",
    "    #最终的反馈是所有步数衰减后的求和\n",
    "    return sum\n",
    "\n",
    "\n",
    "get_value(rs[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/cpu/lib/python3.6/site-packages/numpy/core/fromnumeric.py:3373: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/root/anaconda3/envs/cpu/lib/python3.6/site-packages/numpy/core/_methods.py:170: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[-1.2689316385800076,\n",
       " -1.595738185128587,\n",
       " 0.5337043907456025,\n",
       " 5.91608556615244,\n",
       " nan]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#蒙特卡洛法评估每个状态的价值\n",
    "def get_values_by_monte_carlo(ss, rs):\n",
    "    #记录5个不同开头的价值\n",
    "    #其实只有4个,因为状态4是不可能作为开头状态的\n",
    "    values = [[] for _ in range(5)]\n",
    "\n",
    "    #遍历所有链\n",
    "    for s, r in zip(ss, rs):\n",
    "        #计算不同开头的价值\n",
    "        values[s[0]].append(get_value(r))\n",
    "\n",
    "    #求每个开头的平均价值\n",
    "    return [np.mean(i) for i in values]\n",
    "\n",
    "\n",
    "#-1.228923788722258,-1.6955696284402704,0.4823809701532294,5.967514743019431,0\n",
    "get_values_by_monte_carlo(*get_chains(2000, 20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.11304324114416356"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#计算状态动作对(s,a)出现的频率,以此来估算策略的占用度量\n",
    "def occupancy(ss, rs, s, a):\n",
    "    rho = 0\n",
    "\n",
    "    count_by_time = np.zeros(max_time)\n",
    "    count_by_s_a = np.zeros(max_time)\n",
    "\n",
    "    for si, ri in zip(ss, rs):\n",
    "        for i in range(len(ri)):\n",
    "            s_opt = si[i]\n",
    "            a_opt = si[i + 1]\n",
    "\n",
    "            #统计每个时间步的次数\n",
    "            count_by_time[i] += 1\n",
    "\n",
    "            #统计s，a出现的次数\n",
    "            if s == s_opt and a == a_opt:\n",
    "                count_by_s_a[i] += 1\n",
    "\n",
    "    #i -> [999 - 0]\n",
    "    for i in reversed(range(max_time)):\n",
    "        if count_by_time[i] == 0:\n",
    "            continue\n",
    "\n",
    "        #以时间逐渐衰减\n",
    "        rho += 0.5**i * count_by_s_a[i] / count_by_time[i]\n",
    "\n",
    "    return (1 - 0.5) * rho\n",
    "\n",
    "\n",
    "max_time = 1000\n",
    "ss, rs = get_chains(max_time, 2000)\n",
    "\n",
    "#0.112567796310472\n",
    "occupancy(ss, rs, 3, 1) + occupancy(ss, rs, 3, 2) + occupancy(ss, rs, 3, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.23167624185977734"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#重新定义状态转移概率矩阵\n",
    "P = np.array([\n",
    "    [0.6, 0.4, 0.0, 0.0, 0.0],\n",
    "    [0.3, 0.0, 0.7, 0.0, 0.0],\n",
    "    [0.0, 0.0, 0.0, 0.5, 0.5],\n",
    "    [0.0, 0.18, 0.36, 0.36, 0.1],\n",
    "    [0.0, 0.0, 0.0, 0.0, 0.0],\n",
    "])\n",
    "\n",
    "ss, rs = get_chains(max_time, 2000)\n",
    "\n",
    "#0.23199480615618912\n",
    "occupancy(ss, rs, 3, 1) + occupancy(ss, rs, 3, 2) + occupancy(ss, rs, 3, 3)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第3章-马尔可夫决策过程.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
